package com.xiaotu.spider.project.pageprocessor;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.APageProcessor;
import com.xiaotu.spider.downloader.SpiderHttpClientDownloader;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 JZLYPageProcessor
 * @日期 2017年3月31日
 * @作者 高海军
 * @功能 九州龙渊数据爬取
 */
@Component("JZLYPageProcessor")
public class JZLYPageProcessor extends APageProcessor
{
	
	private static final Map<String, Map<String, String>> DATA_MAP = new HashMap<String, Map<String, String>>();
	
	private static final String[] keyArr = new String[] { "user", "time",
			"content", "sex", "place", "birthday", "shortInfo", "edu",
			"label" };
	
	private static final String CSV_SEP = "|*|";
	
	@Override
	public void process(Page page)
	{
		try
		{
			if (page.getUrl().toString().contains("http://s.weibo.com/weibo")
					|| page.getUrl().toString().contains("http://weibo.com/p/"))
				this.listProcess(page);
			else
				this.detailProcess(page);
		}
		catch (Exception e)
		{
			e.printStackTrace();
			FileUtil.writeFile(
					"D://" + RegexUtils.regexSearch("[0-9]+",
							page.getUrl().toString()) + "_"
							+ e.getClass().getSimpleName() + ".txt",
					StringEscapeUtils
							.unescapeJavaScript(page.getHtml().toString()));
			FileUtil.writeFile("D://err.csv",
					page.getUrl().toString() + SepratorUtil.LINE_SEPRATOR);
			if (page.getRequest().getExtra("key") != null)
			{
				Map<String, String> map = DATA_MAP
						.get(page.getRequest().getExtra("key") + "");
				
				List<Object> dataList = new ArrayList<Object>();
				for (String key : keyArr)
					dataList.add(map.get(key) == null ? "" : map.get(key));
				String line = list2CSVLine(dataList);
				System.out.println(line);
				
				FileUtil.writeFile("D://weibo.csv",
						line + SepratorUtil.LINE_SEPRATOR);
			}
		}
	}
	
	private void detailProcess(Page page)
	{
		String htmlStr = StringEscapeUtils
				.unescapeJavaScript(page.getHtml().toString());
		List<String> labelList = RegexUtils.regexSearchAll(
				"<strong class=\"W_f18\">(.*)</strong><span class=\"S_txt2\">关注</span>|"
						+ "<strong class=\"W_f18\">(.*)</strong><span class=\"S_txt2\">粉丝</span>|"
						+ "<strong class=\"W_f18\">(.*)</strong><span class=\"S_txt2\">微博</span>|"
						+ "<strong class=\"W_f16\">(.*)</strong><span class=\"S_txt2\">关注</span>|"
						+ "<strong class=\"W_f16\">(.*)</strong><span class=\"S_txt2\">粉丝</span>|"
						+ "<strong class=\"W_f16\">(.*)</strong><span class=\"S_txt2\">微博</span>|"
						+ "<strong class=\"W_f14\">(.*)</strong><span class=\"S_txt2\">关注</span>|"
						+ "<strong class=\"W_f14\">(.*)</strong><span class=\"S_txt2\">粉丝</span>|"
						+ "<strong class=\"W_f14\">(.*)</strong><span class=\"S_txt2\">微博</span>",
				htmlStr);
		String read = null, comment = null, fans = null;
		for (String label : labelList)
		{
			List<String> valueList = RegexUtils.regexSearchAll(
					"<strong class=\"(.*?)\">(.*)</strong>|<span class=\"(.*?)\">(.*)</span>",
					label);
			if (valueList == null || valueList.size() < 2)
				continue;
			String value = HtmlUtil.delHTMLTag(valueList.get(0));
			String tag = HtmlUtil.delHTMLTag(valueList.get(1));
			switch (tag)
			{
				case "关注":
					read = value;
					break;
				case "粉丝":
					fans = value;
					break;
				case "微博":
					comment = value;
					break;
			}
		}
		
		if (StringUtils.isEmpty(read) || StringUtils.isEmpty(comment)
				|| StringUtils.isEmpty(fans))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		String sex = "";
		if (htmlStr.contains("icon_pf_female"))
			sex = "女";
		else if (htmlStr.contains("icon_pf_male"))
			sex = "男";
		
		int start = htmlStr.indexOf("<div class=\"PCD_person_info\">");
		int end = htmlStr.substring(start).indexOf("\"})</script>");
		
		String divStr = htmlStr.substring(start, start + end);
		Html divObj = new Html(divStr);
		
		List<String> detailList = divObj
				.xpath("//li[@class='item S_line2 clearfix']").all();
		
		String place = "";
		String birthday = "";
		String shortInfo = "";
		String edu = "";
		String label = "";
		
		for (String detailStr : detailList)
		{
			Html detailObj = new Html(detailStr);
			String value = detailObj.xpath("//li/span[2]/text()").get();
			if (detailStr.contains("ficon_cd_place"))
				place = value;
			else if (detailStr.contains("ficon_constellation"))
				birthday = value;
			else if (detailStr.contains("ficon_pinfo"))
				shortInfo = value;
			else if (detailStr.contains("ficon_cd_coupon"))
				label = StringUtils.join(detailObj
						.xpath("//li/span[2]/a/text()").all().toArray(), ",");
			else if (detailStr.contains("ficon_edu"))
				edu = detailObj.xpath("//li/span[2]/a/text()").get();
		}
		
		Map<String, String> map = DATA_MAP
				.get(page.getRequest().getExtra("key") + "");
		if (map == null)
			map = new HashMap<>();
		map.put("sex", sex.trim());
		map.put("place", place.trim());
		map.put("birthday", birthday.trim());
		map.put("shortInfo", shortInfo.trim().replace("简介：", "").trim());
		map.put("edu", edu.trim());
		map.put("label", label.trim());
		
		List<Object> dataList = new ArrayList<Object>();
		for (String key : keyArr)
			dataList.add(map.get(key));
		String line = list2CSVLine(dataList);
		System.out.println(line);
		
		FileUtil.writeFile("D://weibo.csv", line + SepratorUtil.LINE_SEPRATOR);
	}
	
	/**
	 * 将数据集转成csv中的一行
	 * @param dataList 数据集
	 * @return csv行字符串
	 */
	public String list2CSVLine(List<Object> dataList)
	{
		if (dataList == null || dataList.isEmpty())
			return StringUtils.EMPTY;
		StringBuffer buffer = new StringBuffer();
		String text;
		for (Object data : dataList)
		{
			text = data == null ? null : data.toString();
			if (StringUtils.isEmpty(text))
				buffer.append(CSV_SEP);
			else
				buffer.append(
						CSV_SEP + "\"" + text.replaceAll("\"", "\"\"") + "\"");
		}
		return buffer.toString().substring(CSV_SEP.length());
	}
	
	private void listProcess(Page page)
	{
		String html = StringEscapeUtils
				.unescapeJavaScript(page.getHtml().toString());
		
		if (StringUtils.isEmpty(html))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		int temp = html.indexOf("<div class=\"search_rese clearfix\">");
		int end = html.substring(temp).indexOf("\"})</script>");
		String div = html.substring(html.indexOf("<div class=\"search_feed\">"),
				temp + end);
		
		Html divObj = new Html(div);
		
		List<String> divList = divObj
				.xpath("//div[@class='WB_cardwrap S_bg2 clearfix']").all();
		
		for (String divStr : divList)
		{
			Html oneHtml = new Html(divStr);
			String user = oneHtml.xpath("//div[@class=\"face\"]/a/@title")
					.get();
			String time = oneHtml
					.xpath("//div[@class='feed_from W_textb']/a/@title").get();
			String content = oneHtml.xpath("//p[@class='comment_txt']/text()")
					.get();
			String userId = oneHtml
					.xpath("//a[@class='name_txt W_fb']/@usercard")
					.replace("(id=)|(&usercardkey=weibo_mp)", "").get()
					.replace("&", "?");
			Map<String, String> oneMap = new HashMap<String, String>();
			
			oneMap.put("user", user);
			oneMap.put("time", time);
			oneMap.put("content", content);
			
			DATA_MAP.put(user, oneMap);
			
			Map<String, Object> extMap = new HashMap<String, Object>();
			extMap.put("key", user);
			extMap.putAll(page.getRequest().getExtras());
			extMap.put(SpiderHttpClientDownloader.HEADER_REFERER,
					page.getUrl().toString());
			PageUtils.addTargetRequest(page, "http://weibo.com/u/" + userId,
					extMap);
		}
	}
	
	/**
	 * 设置要爬取的站点地址
	 */
	@SuppressWarnings({ "deprecation" })
	@Override
	protected void setSiteRequest()
	{
		Map<String, Object> tvMap = new HashMap<String, Object>();
		tvMap.put("tvid", "1");
		tvMap.put("tvName", "1");
		
		String[] urlArr = new String[] {
				"http://s.weibo.com/weibo/%25E4%25B9%259D%25E5%25B7%259E%25E9%25BE%2599%25E6%25B8%258A?topnav=1&wvr=6&b=1#1490943475649",
				"http://s.weibo.com/weibo/%25E4%25B9%259D%25E5%25B7%259E%25E9%25BE%2599%25E6%25B8%258A&b=1&page=2",
				"http://s.weibo.com/weibo/%25E4%25B9%259D%25E5%25B7%259E%25E9%25BE%2599%25E6%25B8%258A&b=1&page=3",
				"http://s.weibo.com/weibo/%25E4%25B9%259D%25E5%25B7%259E%25E9%25BE%2599%25E6%25B8%258A&b=1&page=4" };
		for (String url : urlArr)
		{
			Request request = new Request(url);
			request.setExtras(tvMap);
			this.site.addStartRequest(request);
		}
		
	}
	
	@Override
	protected String getSpiderType()
	{
		return "JZLY";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return Constants.PageDataType.WEIBO;
	}
	
	/*
	 * (non-Javadoc)
	 * @see
	 * com.xiaotu.spider.APageProcessor#getStartRequest(net.sf.json.JSONObject)
	 */
	@Override
	protected Request getStartRequest(JSONObject target)
	{
		return null;
	}
}
